{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.75010836,  0.66131496, -0.7249927 ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('Pendulum-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        done = terminated or truncated\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAkpUlEQVR4nO3df3SU1YH/8c8MyQwkYSYkkIkpidDCESmCFRCm7uqppERN/ZnucT0cNrIUKwYL0uWsbJWuPd2GxW+rUhXdbSuetpKW7mIFpZoGDWsNP4wgP00tAonAJALNTBLIr5n7/cNl1lHUBGYyN+H9OmfOkee5c3PnkcybJ/PMxGGMMQIAwELOZC8AAIBPQ6QAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANZKWqSeeOIJjRo1SoMHD9a0adO0bdu2ZC0FAGCppETqN7/5jRYvXqzvf//7euuttzRp0iQVFRWpqakpGcsBAFjKkYwPmJ02bZqmTp2qxx9/XJIUiUSUn5+ve++9V/fff39fLwcAYKmUvv6CnZ2dqq2t1dKlS6PbnE6nCgsLVVNTc9b7dHR0qKOjI/rnSCSikydPKjs7Ww6HI+FrBgDElzFGLS0tysvLk9P56T/U6/NIHT9+XOFwWD6fL2a7z+fTO++8c9b7lJeX66GHHuqL5QEA+lBDQ4NGjhz5qfv7PFLnYunSpVq8eHH0z8FgUAUFBWpoaJDH40niygAA5yIUCik/P19Dhw79zHF9Hqnhw4dr0KBBamxsjNne2Nio3Nzcs97H7XbL7XZ/YrvH4yFSANCPfd5LNn1+dZ/L5dLkyZNVVVUV3RaJRFRVVSW/39/XywEAWCwpP+5bvHixSktLNWXKFF155ZV69NFH1dbWpjlz5iRjOQAASyUlUrfffrs++OADLVu2TIFAQJdffrn+8Ic/fOJiCgDAhS0p75M6X6FQSF6vV8FgkNekAKAf6unzOJ/dBwCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBavY7U5s2bdeONNyovL08Oh0PPP/98zH5jjJYtW6aLLrpIQ4YMUWFhod59992YMSdPntSsWbPk8XiUmZmpuXPnqrW19bweCABg4Ol1pNra2jRp0iQ98cQTZ92/YsUKrVy5Uk899ZS2bt2q9PR0FRUVqb29PTpm1qxZ2rt3ryorK7VhwwZt3rxZd91117k/CgDAwGTOgySzbt266J8jkYjJzc01Dz/8cHRbc3OzcbvdZs2aNcYYY/bt22ckme3bt0fHbNy40TgcDnPkyJEefd1gMGgkmWAweD7LBwAkSU+fx+P6mtTBgwcVCARUWFgY3eb1ejVt2jTV1NRIkmpqapSZmakpU6ZExxQWFsrpdGrr1q1nnbejo0OhUCjmBgAY+OIaqUAgIEny+Xwx230+X3RfIBBQTk5OzP6UlBRlZWVFx3xceXm5vF5v9Jafnx/PZQMALNUvru5bunSpgsFg9NbQ0JDsJQEA+kBcI5WbmytJamxsjNne2NgY3Zebm6umpqaY/d3d3Tp58mR0zMe53W55PJ6YGwBg4ItrpEaPHq3c3FxVVVVFt4VCIW3dulV+v1+S5Pf71dzcrNra2uiYTZs2KRKJaNq0afFcDgCgn0vp7R1aW1v1l7/8JfrngwcPaufOncrKylJBQYEWLVqkH/7whxo7dqxGjx6tBx98UHl5ebrlllskSZdeeqmuu+46zZs3T0899ZS6urq0YMEC/f3f/73y8vLi9sAAAANAby8bfPXVV42kT9xKS0uNMR9ehv7ggw8an89n3G63mTFjhqmrq4uZ48SJE+aOO+4wGRkZxuPxmDlz5piWlpa4X7oIALBTT5/HHcYYk8RGnpNQKCSv16tgMMjrUwDQD/X0ebxfXN0HALgwESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKzVq0iVl5dr6tSpGjp0qHJycnTLLbeorq4uZkx7e7vKysqUnZ2tjIwMlZSUqLGxMWZMfX29iouLlZaWppycHC1ZskTd3d3n/2gAAANKryJVXV2tsrIybdmyRZWVlerq6tLMmTPV1tYWHXPfffdp/fr1Wrt2raqrq3X06FHddttt0f3hcFjFxcXq7OzUG2+8oWeffVarV6/WsmXL4veoAAADgzkPTU1NRpKprq42xhjT3NxsUlNTzdq1a6Nj9u/fbySZmpoaY4wxL730knE6nSYQCETHrFq1yng8HtPR0dGjrxsMBo0kEwwGz2f5AIAk6enz+Hm9JhUMBiVJWVlZkqTa2lp1dXWpsLAwOmbcuHEqKChQTU2NJKmmpkaXXXaZfD5fdExRUZFCoZD27t171q/T0dGhUCgUcwMADHznHKlIJKJFixbpqquu0oQJEyRJgUBALpdLmZmZMWN9Pp8CgUB0zEcDdWb/mX1nU15eLq/XG73l5+ef67IBAP3IOUeqrKxMe/bsUUVFRTzXc1ZLly5VMBiM3hoaGhL+NQEAyZdyLndasGCBNmzYoM2bN2vkyJHR7bm5uers7FRzc3PM2VRjY6Nyc3OjY7Zt2xYz35mr/86M+Ti32y23230uSwUA9GO9OpMyxmjBggVat26dNm3apNGjR8fsnzx5slJTU1VVVRXdVldXp/r6evn9fkmS3+/X7t271dTUFB1TWVkpj8ej8ePHn89jAQAMML06kyorK9Nzzz2n3//+9xo6dGj0NSSv16shQ4bI6/Vq7ty5Wrx4sbKysuTxeHTvvffK7/dr+vTpkqSZM2dq/Pjxmj17tlasWKFAIKAHHnhAZWVlnC0BAGI4jDGmx4MdjrNuf+aZZ3TnnXdK+vDNvN/97ne1Zs0adXR0qKioSE8++WTMj/IOHz6s+fPn67XXXlN6erpKS0u1fPlypaT0rJmhUEher1fBYFAej6enywcAWKKnz+O9ipQtiBQA9G89fR7ns/sAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1UpK9AAD/xxjzqfscDkcfrgSwA5ECLGDCYXW3tCj01ltq3r5d7Q0NCp8+rRSPR+ljxmjY3/yN0r70JQ1KTydWuKAQKSDJIh0dat6yRY3r1+vUu+9KHzmb6vrgA50+cEAnXn1V3iuuUM4ttyjj0ksJFS4YRApIIhOJ6INXXlFg7Vp1Nzd/+rjOTjVv2aL2Y8dUcNddypgwgVDhgsCFE0CSdLe06L1//3cdWb36MwP1Ue2HD6v+P/5Drfv3f+brV8BAQaSAJOhua9PhJ55Qc02NTFdXzL4jbW3a0NCgNe+9pz8ePaq2j+1vP3xYx9asUbi1tS+XDCQFP+4D+pgJh3X4scfUvGVL7HZjdLC1Vd/fsUOHWlvVHg7Lk5qqCcOG6f9NnapU5//9m7Ll7bd16sABDZ00iR/7YUDjTAroQ5HOTh1aufITgZKk91pbNe9Pf9L+YFCnw2EZScGuLv2pqUkLt27Vifb2mPH1Tz3VR6sGkodIAX3EhMNq3rJFLbt2nXX/o3v3KvixH+2dse34cVUePRo736eMBQYSIgX0AWOMTr33nt5/5hl1nTiR7OUA/QaRAvpA++HDOlBeTqCAXiJSQIKdPnxYBx99VF3Hj3/muOL8fKV+ykUQozIyNDErKxHLA6zG1X1AAnWeOKED//Zv6ggEPndsUV6eJOmHb7+tznBYEUmDHA5lulz68dSpujgjI2a8r6QkEUsGrEKkgATpDoX07rJlPQqU9OEHyBbl5WlkWpo2vP++TrS3a1RGhm4fPVrZbnfMWPdFF2mY38/l5xjwiBSQAOFTp9T4wgvqbGrq1f0cDocmDBumCcOGfeqYlGHDlDd7tlI8nvNdJmA9XpMC4izS3a3jf/yjml54QZGOjvhO7nDIc/nlyrzySjkGDYrv3ICFOJMC4uyvr7+u93/xCykSifvcnssvV8G3vy2nyxX3uQEbcSYFxFHz1q16/+c/T0ig0seN08i5c+UcMiTucwO24kwKiANjjILbt+vQI48ofOpU3Od3f+ELGjlnjgaPHMnFErigECkgDoLbt+u98nKZcDjuczsGDdKI665T+rhxBAoXHCIFnAdjjDqbmnT0179OSKAkKW/2bOXcdBOBwgWJSAHnyBij7uZmHf3Vr3T68OG4z+9wuZQ3a5Z8N95IoHDBIlLAOTLd3TpaUaGT1dXxn9zpVG5JiXw33cSl5rigcXUfcA6MMQqsXavjr7ySkPmzZ8yQ7+abCRQueJxJAb0U6ezUsd/+VoH/+i8pAa9DeadNU96sWVxqDohIAb1y5gwq8NvfJmT+1GHDlHPDDXLxieeAJCIF9JgxRq379qlx/fqEzO9ITdXoJUs0dMKEhMwP9EdECugBE4mo7c9/VsN//qciCXizbmpWlkb/0z8p48tfjvvcQH9GpIAe6Dx+XO8/84xOv/de3OdO8XhUcM89nEEBZ8HVfcDniHR26vDKlWrbvz/ucztSUjRyzhx5J0+O+9zAQMCZFPAZuk6e1KHHH1fLrl1xn3tQWppyb79dw66+mkvNgU/BmRTwKcKnT+vw448r9OabCZl/6KRJGl5YKGdqakLmBwYCzqSAszDhsN5bsUKh2tqEzD84P1/58+YpZejQhMwPDBRECviYSGenTm7erLZ33knI/Gljxmjsv/4rv/4d6AF+3Ad8hIlE1LJ7t45VVCjc1hb3+dPGjNGoRYsIFNBDnEkBH9F+5IgOPfaYupub4z63KydHF3/nOxqcnx/3uYGBikgB/+vUgQM6UF6esECNWrhQQy6+mF+7AfQCkcIFzxij9vff18FHH1VnU1NCvkb2jBlKv/RSAgX0EpHCBS/c0qJ3H3xQXSdPxn9yh0PZ114r3623ypnCtxvQW3zX4ILW3dKiY7/5jbr++teEzJ/1ta/p4u98hzMo4BwRKVyQjDEynZ06/vLLOv7HP0rGxP1rZF17rfLnzSNQwHkgUrhgnXjtNR1ds0amqyu+Ezscypw+Xflz5yolPT2+cwMXGCKFC1LzG2+o4emnZbq74z53xoQJyr/rLj5NAogDIoULijFGzW+8ofpVqxISqCFf+pJG3nmnUvnNukBc9OoTJ1atWqWJEyfK4/HI4/HI7/dr48aN0f3t7e0qKytTdna2MjIyVFJSosbGxpg56uvrVVxcrLS0NOXk5GjJkiXqTsCTBfBxZwJ18Mc/VncoFPf5nUOGaERRkdLGjOF1KCBOehWpkSNHavny5aqtrdWbb76pa6+9VjfffLP27t0rSbrvvvu0fv16rV27VtXV1Tp69Khuu+226P3D4bCKi4vV2dmpN954Q88++6xWr16tZcuWxfdRAWfREQjo8E9/mpAzKDmdGnnnnRpx3XUECogjhzHnd1lTVlaWHn74YX3zm9/UiBEj9Nxzz+mb3/ymJOmdd97RpZdeqpqaGk2fPl0bN27UN77xDR09elQ+n0+S9NRTT+mf//mf9cEHH8jlcvXoa4ZCIXm9XgWDQXn4DDR8DmOMuk6e1Ps//7n++qc/xf9KPqdT+d/6lkbccIMcTj4OE+iJnj6Pn/N3VDgcVkVFhdra2uT3+1VbW6uuri4VFhZGx4wbN04FBQWqqamRJNXU1Oiyyy6LBkqSioqKFAqFomdjZ9PR0aFQKBRzA3rCGKNwW5uO/vKXCQmU0+3WF0pLPzyDIlBA3PX6u2r37t3KyMiQ2+3W3XffrXXr1mn8+PEKBAJyuVzKzMyMGe/z+RQIBCRJgUAgJlBn9p/Z92nKy8vl9Xqjt3w+oBO9cKyiQic2bUrIe6FG3HCDRlx/vRx8mgSQEL2O1CWXXKKdO3dq69atmj9/vkpLS7Vv375ErC1q6dKlCgaD0VtDQ0NCvx4Ghkh3t47+8pf64MUXEzL/sKuvVm5JiQYNHpyQ+QGcwyXoLpdLY8aMkSRNnjxZ27dv12OPPabbb79dnZ2dam5ujjmbamxsVG5uriQpNzdX27Zti5nvzNV/Z8acjdvtltvt7u1ScQGLdHbqyK9+pabnn0/I/K6cHOXceKMG8V4oIKHO+4fokUhEHR0dmjx5slJTU1VVVRXdV1dXp/r6evn9fkmS3+/X7t271fSRT5qurKyUx+PR+PHjz3cpQFRo586EBSrF69WoRYuUccklXMkHJFivzqSWLl2q66+/XgUFBWppadFzzz2n1157TS+//LK8Xq/mzp2rxYsXKysrSx6PR/fee6/8fr+mT58uSZo5c6bGjx+v2bNna8WKFQoEAnrggQdUVlbGmRLiwkQianv3Xb2/enVC5nempWnMsmVKHzs2IfMDiNWrSDU1Nekf/uEfdOzYMXm9Xk2cOFEvv/yyvv71r0uSHnnkETmdTpWUlKijo0NFRUV68skno/cfNGiQNmzYoPnz58vv9ys9PV2lpaX6wQ9+EN9HhQuSMUanDx1Sw9NPq+P99+M+f+rw4Rq9eLHS/vfH3QAS77zfJ5UMvE8KZxM+fVp/eeghtSbgQh5nWpouvuceDbvqKjkGDYr7/MCFpqfP41w3iwGhu7VVB370o4QEyuF2a2RpqYZ99asECuhjRAr9XueJEzq8cqVa9+xJyPwjZs5U9rXX8l4oIAn4rkO/Funs1KFHH1XL228nZP70ceOU841vyMmFPUBSECn0W5GuLp149VW17NqVkPmHfPGLGrNsmVIyMhIyP4DPR6TQL5nubjVv3aqjv/pVQj7uKH3cOH3p/vsJFJBkfCIm+h1jjNoOHNCR1avVHQzGff4hF1+sgm9/WynDhsV9bgC9w5kU+p32hgb95aGHFG5tjfvcrpwcXfyd72jIF7/Ip0kAFiBS6Fda6+p06Mc/TkigJClv1ix+sy5gESKFfsEYo44jR3ToJz9Rx2f8Wpdz5nBo+Ne/rsxp0wgUYBEihX4h3NamY7/7nTqOHYv/5A6HRlx/vfK/9S3eCwVYhu9IWC986pQan39eJ6urEzL/8JkzlT9vHp8mAViISMFqke5unaiqUtP69VI4HN/JHQ5lXX21vjB7NoECLEWkYC1jjFr37VPDz36WkPdCeadO1cg5c/jFhYDFiBSs1R0K6VhFRWLerHvJJRo5Z45Ss7LiPjeA+OHNvLCSiUQU+O1vE/Op5qmp8t16q9x5eXGfG0B8cSYFK50+dEihXbukSCSu8zrT0pQ/b54y/X4uNQf6ASIFK506cEDthw/Hfd78b31Lw2fMiPu8ABKDSOGC4HC5NHLOHGV/7WvJXgqAXuA1KQx4DpdLF/3d32l4YaHk5K880J9wJoUBL/vaazWiuJhfXAj0Q/yzElZyut1yuFznPU+K16vckhJ+LxTQTxEpWMkzebI8Eyee1xzuvDx96Xvfk9vni9OqAPQ1IgUrpaSna/jMmXIOHnxO9x+UkaHRixcrY9y4OK8MQF8iUrCWd+pU+UpKevVakiMlRS6fT5f86EdKGzs2gasD0Be4cALWcgwaJN9NN8l0d+uDF1/8/F906HRq2NVXK++OO/gRHzBAEClYbdCQIfLdcosGjxypYxUV6jh69JOf5edwKMXrVc4NNyh75ky5+Dw+YMAgUrBeSnq6sv72b+X9ylcU2rFDoV271HHsmEw4rNTsbA398pflnTJFruHDeR8UMMAQKfQLDqdTKR6Psq65RlnXXJPs5QDoI/yzEwBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWOu8IrV8+XI5HA4tWrQouq29vV1lZWXKzs5WRkaGSkpK1NjYGHO/+vp6FRcXKy0tTTk5OVqyZIm6u7vPZykAgAHonCO1fft2Pf3005o4cWLM9vvuu0/r16/X2rVrVV1draNHj+q2226L7g+HwyouLlZnZ6feeOMNPfvss1q9erWWLVt27o8CADAwmXPQ0tJixo4dayorK80111xjFi5caIwxprm52aSmppq1a9dGx+7fv99IMjU1NcYYY1566SXjdDpNIBCIjlm1apXxeDymo6OjR18/GAwaSSYYDJ7L8gEASdbT5/FzOpMqKytTcXGxCgsLY7bX1taqq6srZvu4ceNUUFCgmpoaSVJNTY0uu+wy+Xy+6JiioiKFQiHt3bv3rF+vo6NDoVAo5gYAGPhSenuHiooKvfXWW9q+ffsn9gUCAblcLmVmZsZs9/l8CgQC0TEfDdSZ/Wf2nU15ebkeeuih3i4VANDP9epMqqGhQQsXLtSvf/1rDR48OFFr+oSlS5cqGAxGbw0NDX32tQEAydOrSNXW1qqpqUlXXHGFUlJSlJKSourqaq1cuVIpKSny+Xzq7OxUc3NzzP0aGxuVm5srScrNzf3E1X5n/nxmzMe53W55PJ6YGwBg4OtVpGbMmKHdu3dr586d0duUKVM0a9as6H+npqaqqqoqep+6ujrV19fL7/dLkvx+v3bv3q2mpqbomMrKSnk8Ho0fPz5ODwsAMBD06jWpoUOHasKECTHb0tPTlZ2dHd0+d+5cLV68WFlZWfJ4PLr33nvl9/s1ffp0SdLMmTM1fvx4zZ49WytWrFAgENADDzygsrIyud3uOD0sAMBA0OsLJz7PI488IqfTqZKSEnV0dKioqEhPPvlkdP+gQYO0YcMGzZ8/X36/X+np6SotLdUPfvCDeC8FANDPOYwxJtmL6K1QKCSv16tgMMjrUwDQD/X0eZzP7gMAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWCsl2Qs4F8YYSVIoFErySgAA5+LM8/eZ5/NP0y8jdeLECUlSfn5+klcCADgfLS0t8nq9n7q/X0YqKytLklRfX/+ZD+5CFwqFlJ+fr4aGBnk8nmQvx1ocp57hOPUMx6lnjDFqaWlRXl7eZ47rl5FyOj98Kc3r9fKXoAc8Hg/HqQc4Tj3DceoZjtPn68lJBhdOAACsRaQAANbql5Fyu936/ve/L7fbneylWI3j1DMcp57hOPUMxym+HObzrv8DACBJ+uWZFADgwkCkAADWIlIAAGsRKQCAtfplpJ544gmNGjVKgwcP1rRp07Rt27ZkL6lPbd68WTfeeKPy8vLkcDj0/PPPx+w3xmjZsmW66KKLNGTIEBUWFurdd9+NGXPy5EnNmjVLHo9HmZmZmjt3rlpbW/vwUSRWeXm5pk6dqqFDhyonJ0e33HKL6urqYsa0t7errKxM2dnZysjIUElJiRobG2PG1NfXq7i4WGlpacrJydGSJUvU3d3dlw8loVatWqWJEydG33jq9/u1cePG6H6O0dktX75cDodDixYtim7jWCWI6WcqKiqMy+Uyv/jFL8zevXvNvHnzTGZmpmlsbEz20vrMSy+9ZL73ve+Z//7v/zaSzLp162L2L1++3Hi9XvP888+bt99+29x0001m9OjR5vTp09Ex1113nZk0aZLZsmWL+Z//+R8zZswYc8cdd/TxI0mcoqIi88wzz5g9e/aYnTt3mhtuuMEUFBSY1tbW6Ji7777b5Ofnm6qqKvPmm2+a6dOnm69+9avR/d3d3WbChAmmsLDQ7Nixw7z00ktm+PDhZunSpcl4SAnxwgsvmBdffNH8+c9/NnV1deZf/uVfTGpqqtmzZ48xhmN0Ntu2bTOjRo0yEydONAsXLoxu51glRr+L1JVXXmnKysqifw6HwyYvL8+Ul5cncVXJ8/FIRSIRk5ubax5++OHotubmZuN2u82aNWuMMcbs27fPSDLbt2+Pjtm4caNxOBzmyJEjfbb2vtTU1GQkmerqamPMh8ckNTXVrF27Njpm//79RpKpqakxxnz4jwGn02kCgUB0zKpVq4zH4zEdHR19+wD60LBhw8zPfvYzjtFZtLS0mLFjx5rKykpzzTXXRCPFsUqcfvXjvs7OTtXW1qqwsDC6zel0qrCwUDU1NUlcmT0OHjyoQCAQc4y8Xq+mTZsWPUY1NTXKzMzUlClTomMKCwvldDq1devWPl9zXwgGg5L+78OJa2tr1dXVFXOcxo0bp4KCgpjjdNlll8nn80XHFBUVKRQKae/evX24+r4RDodVUVGhtrY2+f1+jtFZlJWVqbi4OOaYSPx9SqR+9QGzx48fVzgcjvmfLEk+n0/vvPNOklZll0AgIElnPUZn9gUCAeXk5MTsT0lJUVZWVnTMQBKJRLRo0SJdddVVmjBhgqQPj4HL5VJmZmbM2I8fp7MdxzP7Bordu3fL7/ervb1dGRkZWrduncaPH6+dO3dyjD6ioqJCb731lrZv3/6Jffx9Spx+FSngXJSVlWnPnj16/fXXk70UK11yySXauXOngsGgfve736m0tFTV1dXJXpZVGhoatHDhQlVWVmrw4MHJXs4FpV/9uG/48OEaNGjQJ66YaWxsVG5ubpJWZZczx+GzjlFubq6amppi9nd3d+vkyZMD7jguWLBAGzZs0KuvvqqRI0dGt+fm5qqzs1PNzc0x4z9+nM52HM/sGyhcLpfGjBmjyZMnq7y8XJMmTdJjjz3GMfqI2tpaNTU16YorrlBKSopSUlJUXV2tlStXKiUlRT6fj2OVIP0qUi6XS5MnT1ZVVVV0WyQSUVVVlfx+fxJXZo/Ro0crNzc35hiFQiFt3bo1eoz8fr+am5tVW1sbHbNp0yZFIhFNmzatz9ecCMYYLViwQOvWrdOmTZs0evTomP2TJ09WampqzHGqq6tTfX19zHHavXt3TNArKyvl8Xg0fvz4vnkgSRCJRNTR0cEx+ogZM2Zo9+7d2rlzZ/Q2ZcoUzZo1K/rfHKsESfaVG71VUVFh3G63Wb16tdm3b5+56667TGZmZswVMwNdS0uL2bFjh9mxY4eRZH7yk5+YHTt2mMOHDxtjPrwEPTMz0/z+9783u3btMjfffPNZL0H/yle+YrZu3Wpef/11M3bs2AF1Cfr8+fON1+s1r732mjl27Fj0durUqeiYu+++2xQUFJhNmzaZN9980/j9fuP3+6P7z1wyPHPmTLNz507zhz/8wYwYMWJAXTJ8//33m+rqanPw4EGza9cuc//99xuHw2FeeeUVYwzH6LN89Oo+YzhWidLvImWMMT/96U9NQUGBcblc5sorrzRbtmxJ9pL61KuvvmokfeJWWlpqjPnwMvQHH3zQ+Hw+43a7zYwZM0xdXV3MHCdOnDB33HGHycjIMB6Px8yZM8e0tLQk4dEkxtmOjyTzzDPPRMecPn3a3HPPPWbYsGEmLS3N3HrrrebYsWMx8xw6dMhcf/31ZsiQIWb48OHmu9/9runq6urjR5M4//iP/2guvvhi43K5zIgRI8yMGTOigTKGY/RZPh4pjlVi8Ks6AADW6levSQEALixECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWOv/AxltgobiwAPPAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.1046]], grad_fn=<MulBackward0>)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "class Model(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.sequential = torch.nn.Sequential(\n",
    "            torch.nn.Linear(3, 64),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(64, 64),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(64, 1),\n",
    "            torch.nn.Tanh(),\n",
    "        )\n",
    "\n",
    "    def forward(self, state):\n",
    "        return self.sequential(state) * 2.0\n",
    "\n",
    "\n",
    "model_action = Model()\n",
    "\n",
    "model_action_next = Model()# 这个模型做延迟更新\n",
    "\n",
    "model_action_next.load_state_dict(model_action.state_dict())\n",
    "\n",
    "model_action(torch.randn(1, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0052]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 输入4维度[state,action]评估的是在一个状态下执行一个状态的分数\n",
    "model_value = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 1),\n",
    ")\n",
    "model_value_next = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 1),\n",
    ")\n",
    "\n",
    "model_value_next.load_state_dict(model_value.state_dict())\n",
    "\n",
    "model_value(torch.randn(1, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5795311396026129"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "def get_action(state):\n",
    "    state = torch.FloatTensor(state).reshape(1, 3)\n",
    "    action = model_action(state).item()\n",
    "    #给动作添加噪声,增加探索\n",
    "    action += random.normalvariate(mu=0, sigma=0.01)\n",
    "    return action\n",
    "\n",
    "\n",
    "get_action([1, 2, 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200,\n",
       " (array([-0.812337  , -0.58318835,  0.6030439 ], dtype=float32),\n",
       "  -0.26770441461485023,\n",
       "  -6.381519426847657,\n",
       "  array([-0.8086616 , -0.5882741 ,  0.12549697], dtype=float32),\n",
       "  False))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#样本池\n",
    "datas = []\n",
    "\n",
    "\n",
    "#向样本池中添加N条数据,删除M条最古老的数据\n",
    "def update_data():\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        action = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        next_state, reward, over, _ = env.step([action])\n",
    "\n",
    "        #记录数据样本\n",
    "        datas.append((state, action, reward, next_state, over))\n",
    "\n",
    "        #更新游戏状态,开始下一个动作\n",
    "        state = next_state\n",
    "\n",
    "    #数据上限,超出时从最古老的开始删除\n",
    "    while len(datas) > 10000:\n",
    "        datas.pop(0)\n",
    "\n",
    "\n",
    "update_data()\n",
    "\n",
    "len(datas), datas[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\cgq10\\AppData\\Local\\Temp\\ipykernel_31284\\1710091499.py:7: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at C:\\actions-runner\\_work\\pytorch\\pytorch\\builder\\windows\\pytorch\\torch\\csrc\\utils\\tensor_new.cpp:248.)\n",
      "  state = torch.FloatTensor([i[0] for i in samples]).reshape(-1, 3)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[-0.5882,  0.8087, -2.3345],\n",
       "         [-0.9979,  0.0646, -2.6867],\n",
       "         [ 0.9999, -0.0126, -5.1404],\n",
       "         [ 0.7138, -0.7004, -1.2842],\n",
       "         [-0.9506, -0.3105, -2.0579]]),\n",
       " tensor([[-0.3790],\n",
       "         [-0.4614],\n",
       "         [-1.1686],\n",
       "         [-0.5724],\n",
       "         [-0.4153]]),\n",
       " tensor([[ -5.3836],\n",
       "         [-10.1897],\n",
       "         [ -2.6439],\n",
       "         [ -0.7673],\n",
       "         [ -8.4095]]),\n",
       " tensor([[-0.5138,  0.8579, -1.7848],\n",
       "         [-0.9801,  0.1987, -2.7075],\n",
       "         [ 0.9614, -0.2752, -5.3251],\n",
       "         [ 0.6443, -0.7648, -1.8953],\n",
       "         [-0.9805, -0.1967, -2.3531]]),\n",
       " tensor([[0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0]]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取一批数据样本\n",
    "def get_sample():\n",
    "    #从样本池中采样\n",
    "    samples = random.sample(datas, 64)\n",
    "\n",
    "    #[b, 3]\n",
    "    state = torch.FloatTensor([i[0] for i in samples]).reshape(-1, 3)\n",
    "    #[b, 1]\n",
    "    action = torch.FloatTensor([i[1] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 1]\n",
    "    reward = torch.FloatTensor([i[2] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 3]\n",
    "    next_state = torch.FloatTensor([i[3] for i in samples]).reshape(-1, 3)\n",
    "    #[b, 1]\n",
    "    over = torch.LongTensor([i[4] for i in samples]).reshape(-1, 1)\n",
    "\n",
    "    return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "state[:5], action[:5], reward[:5], next_state[:5], over[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1349.6323278402963"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "\n",
    "\n",
    "def test(play):\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #记录反馈值的和,这个值越大越好\n",
    "    reward_sum = 0\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        action = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        state, reward, over, _ = env.step([action])\n",
    "        reward_sum += reward\n",
    "\n",
    "        #打印动画\n",
    "        if play and random.random() < 0.2:  #跳帧\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    return reward_sum\n",
    "\n",
    "\n",
    "test(play=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[-0.1103],\n",
       "         [-0.1399],\n",
       "         [-0.1524],\n",
       "         [-0.0263],\n",
       "         [-0.1088]], grad_fn=<SliceBackward0>),\n",
       " tensor([[ -5.4601],\n",
       "         [-10.3245],\n",
       "         [ -2.8111],\n",
       "         [ -0.8174],\n",
       "         [ -8.5307]], grad_fn=<SliceBackward0>))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_value(state, action):\n",
    "    #直接评估综合了state和action的value\n",
    "    #[b, 3+1] -> [b, 4]\n",
    "    input = torch.cat([state, action], dim=1)\n",
    "\n",
    "    #[b, 4] -> [b, 1]\n",
    "    return model_value(input)\n",
    "\n",
    "\n",
    "def get_target(next_state, reward, over):\n",
    "    #对next_state的评估需要先把它对应的动作计算出来,这里用model_action_next来计算\n",
    "    #[b, 3] -> [b, 1]\n",
    "    action = model_action_next(next_state)\n",
    "\n",
    "    #和value的计算一样,action拼合进next_state里综合计算\n",
    "    #[b, 3+1] -> [b, 4]\n",
    "    input = torch.cat([next_state, action], dim=1)\n",
    "\n",
    "    #[b, 4] -> [b, 1]\n",
    "    target = model_value_next(input) * 0.98\n",
    "\n",
    "    #[b, 1] * [b, 1] -> [b, 1]\n",
    "    target *= (1 - over)\n",
    "\n",
    "    #[b, 1] + [b, 1] -> [b, 1]\n",
    "    target += reward\n",
    "\n",
    "    return target\n",
    "\n",
    "\n",
    "get_value(state, action)[:5], get_target(next_state, reward, over)[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.1462, grad_fn=<NegBackward0>)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_loss_action(state):\n",
    "    #首先把动作计算出来\n",
    "    #[b, 3] -> [b, 1]\n",
    "    action = model_action(state)\n",
    "\n",
    "    #像value计算那里一样,拼合state和action综合计算\n",
    "    #[b, 3+1] -> [b, 4]\n",
    "    input = torch.cat([state, action], dim=1)\n",
    "\n",
    "    #使用value网络评估动作的价值,价值是越高越好\n",
    "    \"\"\"因为这里是在计算loss,loss是越小越好,所以符号取反！！！！！！\"\"\"\n",
    "    #[b, 4] -> [b, 1] -> [1]\n",
    "    loss = -model_value(input).mean()  \n",
    "\n",
    "    return loss\n",
    "\n",
    "\n",
    "get_loss_action(state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def soft_update(model, model_next):\n",
    "    for param, param_next in zip(model.parameters(), model_next.parameters()):\n",
    "        #以一个小的比例更新\n",
    "        value = param_next.data * 0.995 + param.data * 0.005\n",
    "        param_next.data.copy_(value)\n",
    "\n",
    "\n",
    "soft_update(torch.nn.Linear(4, 64), torch.nn.Linear(4, 64))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 400 -1133.4190528762065\n",
      "20 4400 -611.8656345407627\n",
      "40 8400 -292.5472179990497\n",
      "60 10000 -209.04403687043677\n",
      "80 10000 -159.60666920648413\n",
      "100 10000 -164.69361791313005\n",
      "120 10000 -108.8726953110158\n",
      "140 10000 -145.12584666619594\n",
      "160 10000 -193.4704736501232\n",
      "180 10000 -349.0929408221572\n"
     ]
    }
   ],
   "source": [
    "def train():\n",
    "    model_action.train()\n",
    "    model_value.train()\n",
    "    optimizer_action = torch.optim.Adam(model_action.parameters(), lr=5e-4)\n",
    "    optimizer_value = torch.optim.Adam(model_value.parameters(), lr=5e-3)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #训练N次\n",
    "    for epoch in range(200):\n",
    "        #更新N条数据\n",
    "        update_data()\n",
    "\n",
    "        #每次更新过数据后,学习N次\n",
    "        for i in range(200):\n",
    "            #采样一批数据\n",
    "            state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "            #计算value和target\n",
    "            value = get_value(state, action)\n",
    "            target = get_target(next_state, reward, over)\n",
    "\n",
    "            #两者求差,计算loss,更新参数\n",
    "            loss_value = loss_fn(value, target)\n",
    "\n",
    "            optimizer_value.zero_grad()\n",
    "            loss_value.backward()\n",
    "            optimizer_value.step()\n",
    "\n",
    "            #使用value网络评估action网络的loss,更新参数\n",
    "            loss_action = get_loss_action(state)\n",
    "\n",
    "            optimizer_action.zero_grad()\n",
    "            loss_action.backward()\n",
    "            optimizer_action.step()\n",
    "\n",
    "            #以一个小的比例更新\n",
    "            soft_update(model_action, model_action_next)\n",
    "            soft_update(model_value, model_value_next)\n",
    "\n",
    "        if epoch % 20 == 0:\n",
    "            test_result = sum([test(play=False) for _ in range(10)]) / 10\n",
    "            print(epoch, len(datas), test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAlEUlEQVR4nO3df3DU9YH/8df+SJaEZDckkIQMiWClYsoPFRT26tVeSYk0tlrj1XM4ZSxnRy84Ih3n5E5x6twMfOW+Z/VO8WY6p/bukB53Yk8ENd8AsUoMGKGGCLn6lTZR2IQfzW6CZJPsvr9/eOzXYLAJfLL73uT5mNkZs5/3vvP+fIR9stlPPusyxhgBAGAhd6oXAADA+RApAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1Uhapp59+WtOnT9eECRO0cOFC7d27N1VLAQBYKiWR+sUvfqHVq1fr0Ucf1Xvvvad58+apsrJSnZ2dqVgOAMBSrlRcYHbhwoW65ppr9I//+I+SpHg8rtLSUt1333166KGHkr0cAIClvMn+hn19fWpqatKaNWsS97ndblVUVKihoWHIx0SjUUWj0cTX8Xhcp06dUkFBgVwu16ivGQDgLGOMuru7VVJSIrf7/D/US3qkTpw4oVgspqKiokH3FxUV6fDhw0M+Zt26dfrJT36SjOUBAJKovb1d06ZNO+/2pEfqQqxZs0arV69OfB0Oh1VWVqb29nb5/f4UrgwAcCEikYhKS0uVm5v7peOSHqnJkyfL4/Goo6Nj0P0dHR0qLi4e8jE+n08+n+8L9/v9fiIFAGnsD71lk/Sz+zIzMzV//nzV1dUl7ovH46qrq1MwGEz2cgAAFkvJj/tWr16t5cuXa8GCBbr22mv105/+VKdPn9Zdd92ViuUAACyVkkjddtttOn78uNauXatQKKQrr7xSr7322hdOpgAAjG8p+T2pixWJRBQIBBQOh3lPCgDS0HCfx7l2HwDAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrjThSb775pr773e+qpKRELpdLL7/88qDtxhitXbtWU6dOVVZWlioqKvSb3/xm0JhTp05p2bJl8vv9ysvL04oVK9TT03NROwIAGHtGHKnTp09r3rx5evrpp4fc/vjjj+upp57Ss88+q8bGRk2cOFGVlZXq7e1NjFm2bJlaWlpUW1urbdu26c0339SPfvSjC98LAMDYZC6CJLN169bE1/F43BQXF5sNGzYk7uvq6jI+n8+8+OKLxhhjPvjgAyPJ7Nu3LzFmx44dxuVymU8++WRY3zccDhtJJhwOX8zyAQApMtzncUffkzpy5IhCoZAqKioS9wUCAS1cuFANDQ2SpIaGBuXl5WnBggWJMRUVFXK73WpsbBxy3mg0qkgkMugGABj7HI1UKBSSJBUVFQ26v6ioKLEtFAqpsLBw0Hav16v8/PzEmHOtW7dOgUAgcSstLXVy2QAAS6XF2X1r1qxROBxO3Nrb21O9JABAEjgaqeLiYklSR0fHoPs7OjoS24qLi9XZ2Tlo+8DAgE6dOpUYcy6fzye/3z/oBgAY+xyN1IwZM1RcXKy6urrEfZFIRI2NjQoGg5KkYDCorq4uNTU1Jcbs3LlT8XhcCxcudHI5AIA05x3pA3p6evThhx8mvj5y5IgOHDig/Px8lZWVadWqVfrbv/1bzZw5UzNmzNAjjzyikpIS3XzzzZKkK664QjfccIPuvvtuPfvss+rv79fKlSv1Z3/2ZyopKXFsxwAAY8BITxvctWuXkfSF2/Lly40xn52G/sgjj5iioiLj8/nM4sWLTWtr66A5Tp48aW6//XaTk5Nj/H6/ueuuu0x3d7fjpy4CAOw03OdxlzHGpLCRFyQSiSgQCCgcDvP+FACkoeE+j6fF2X0AgPGJSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWGvFV0AEMn4nH1d/Vpch776m7uVl9//NZapmFhcqdM0f+q69WRl6eXG7+vQgMhUgBo2QgEtHv335bx7ZsUf/Jk9Lnr+Xc0qJTu3cro6BAU2+9VZOuu05eLpYMfAGRAkZBf1eXjm3ZopNvvKF4NDr0IGPUf+KE2p97Tmc++URT//RPlZGXl9R1ArbjZwyAw+J9fTq2ZYtOvP76+QP1OSYa1YnXX9exf/93xfv6krBCIH0QKcBhp3bv1vFXX5UZQXBMX5+Ob9+uk7t2jeLKgPRDpAAHDUQi6nz1VSkeH/mD43Edf/VVDUQizi8MSFNECnDQibo69ba3X/Djez/+WJ3btine3+/gqoD0RaQAB8U//VRmYOCCH28GBnRs82b1HDwo8/mzAYFxikgBFuo7dSrVSwCsQKQAC/UdPz7496qAcYpIARbqaWm5sJMvgDGGSAEW6jl0SIZIAUQKcJI7K0sujyfVywDGDCIFOMh/5ZXKmDLFkbk4uw8gUoCjvLm5cmdmXvxExmigu/vi5wHSHJECHOT1+x2L1NmP9QDGMyIFOMjt8znznpQxn328BzDOESnAQiYWU9c776R6GUDKESnAUgNdXaleApByRApwWE55ueRypXoZwJhApACH+efPdyRSJhbjQxAx7hEpwGG+KVMciVQ8GuU0dIx7RApwmMfvlxM/7Iv39SnW0+PATED6IlKAw1xuZ/5aRY8dU/f77zsyF5CuiBRgKTMwoFhvb6qXAaQUkQIc5vJ4NPGKK1K9DGBMIFKAw1wej3JmzXJmMmO40CzGNSIFOM3lUubkyY5MNdDdLdPf78hcQDoiUoDTXC558/MdmWogEiFSGNeIFDAKHLkSuqS+48c5eQLjGpECHOZy8JJIPQcPqv/3v3dsPiDdECkAgLWIFDAKJkybppzZs52ZjDP8MI4RKWAUeLKy5A0EHJkrfuaMI/MA6YhIAaPA7fPJm5PjyFx9J044Mg+QjogUMApcGRlyT5jgyFzRzk5H5gHSEZECRoHL5XLsgw+7339f4j0pjFNECrDcpx99RKQwbhEpYJRk5OXJlZGR6mUAaY1IAaPEf9VVysjLc2QuMzDgyDxAuiFSwCjxBgLOvJKKx9Xf1XXx8wBpiEgBo8Tr98vl9V70PMYYTkPHuEWkgFHi9noduY6f6e9X+N13HVgRkH6IFGC7eFy9bW2pXgWQEkQKGEW+kpJULwFIa0QKGEWBa691bC4uMovxiEgBoyhzyhRH5on39SkejToyF5BOiBQwijILCx25PFK8t1exnh4HVgSkFyIFjCJvdrYj88R6ezVw+rQjcwHphEgBo8mhi8z2trfrdEuLI3MB6YRIAekgHpeJxVK9CiDpiBQwilwZGcq98krH5uMMP4w3I4rUunXrdM011yg3N1eFhYW6+eab1draOmhMb2+vampqVFBQoJycHFVXV6ujo2PQmLa2NlVVVSk7O1uFhYV68MEHNcAFNDEGubxeZV96qSNzxQcG+MgOjDsjilR9fb1qamr0zjvvqLa2Vv39/VqyZIlOf+4N3QceeECvvPKKtmzZovr6eh09elS33HJLYnssFlNVVZX6+vq0Z88evfDCC3r++ee1du1a5/YKsITL7VZmQYEjcw2Ew1wNHeOOy1zEzw+OHz+uwsJC1dfX6xvf+IbC4bCmTJmiTZs26dZbb5UkHT58WFdccYUaGhq0aNEi7dixQzfeeKOOHj2qoqIiSdKzzz6rv/qrv9Lx48eVmZn5B79vJBJRIBBQOByW3++/0OUDo84Yo9+/9ZaObNhw0XPlL16ssrvvlsehMwaBVBru8/hFvScVDoclSfn5+ZKkpqYm9ff3q6KiIjFm1qxZKisrU0NDgySpoaFBc+bMSQRKkiorKxWJRNRynrOXotGoIpHIoBuQDlwul1wejyNzRT/+WLHeXkfmAtLFBUcqHo9r1apV+vrXv67Zs2dLkkKhkDIzM5V3zge9FRUVKRQKJcZ8PlBnt5/dNpR169YpEAgkbqWlpRe6bCBtnW5tVYzflcI4c8GRqqmp0cGDB7V582Yn1zOkNWvWKBwOJ27t7e2j/j0Bp7jcbsnNibTAhbigvzkrV67Utm3btGvXLk2bNi1xf3Fxsfr6+tR1zqeIdnR0qLi4ODHm3LP9zn59dsy5fD6f/H7/oBuQLrIuvVQ5X/uaM5PF45yGjnFlRJEyxmjlypXaunWrdu7cqRkzZgzaPn/+fGVkZKiuri5xX2trq9ra2hQMBiVJwWBQzc3N6uzsTIypra2V3+9XeXn5xewLYCVPVpa8Eyc6Mlc/78dinBnRZ1vX1NRo06ZN+uUvf6nc3NzEe0iBQEBZWVkKBAJasWKFVq9erfz8fPn9ft13330KBoNatGiRJGnJkiUqLy/XHXfcoccff1yhUEgPP/ywampq5PP5nN9DIMXcEybI41Ck+o4fd2QeIF2M6JXUxo0bFQ6H9c1vflNTp05N3H7xi18kxjzxxBO68cYbVV1drW984xsqLi7WSy+9lNju8Xi0bds2eTweBYNB/fmf/7nuvPNOPfbYY87tFWARl9crV0aGI3P1HDzoyDxAurio35NKFX5PCunmd888oxOvvXbR83gDAc39+c/lcujCtUCqJOX3pAAMj9vnc+yK6MB4QqSAJAgsWCBvbm6qlwGkHSIFJEFGXp4zV54wRvG+voufB0gTRApIgoxJk+Tyjuhk2iGZeFz9p045sCIgPRApIAk8OTmOvJIysRiRwrhCpIAkcLndjpw4EY9GFd6714EVAemBSAHpJB5X34kTqV4FkDRECkgSpz6hFxhPiBSQJP75852ZyBguMotxg0gBSeI753PULlS8t1fxM2ccmQuwHZECkiSjoMCReWJnzvDhhxg3iBSQJG6HrvIfO31aAz09jswF2I5IAWnmzG9/q08//DDVywCSgkgB6YgTJzBOECkgSTxZWQosXOjYfJzhh/GASAFJ4srI0IRp0xyZK9bby6spjAsXf8VLAMPicruVMWnSsMd/fPq0/vN3v1OWx6NLcnL0tbw8lWRny+1yaaCrSyYW++xyS8AYRqSAZHG75Zk4cdjDC3w+VZaU6FQ0qv/b3a3dx46pMCtLN5aWKufYMZn+fsmhj6UHbEWkgCRxuVwj+tj3CR6PLg8EJEnBwkINGKN9J05o80cfqfHnP9eD1dWa+pWv8FHyGNP4WQFgqbNRO3vLcLsVnDJF91x+uVyS/tff/Z1OnTrFCRQY04gUkERZl1yiCZdccsGPd7lcmpKVpT+dPl1zv/pVrV+/nlBhTCNSQBJ5cnPlzc296HkyPR5V33STCgsLtWXLFg0MDDiwOsA+RApIIk92tjxZWc7M1durH/zgB9q3b58+/vhjR+YEbEOkgCTyZGXJk53tyFx9nZ0qKyvT0qVL9a//+q/8yA9jEpECksi4XDJ/4Gy8T06f1rb2dr340Uf6P0eP6nR//5DjIvv3S5JuuOEGNTc3q7Oz0/H1AqnGKehAksRiMR08eFBd7e3KGWK7MUZHenr06P79+m1Pj3pjMfkzMjR70iT93TXXKOOcX9ztaWmRJE2cOFHz5s1TU1OTvvOd7yRhT4Dk4ZUUkCSnT5/Whg0b1HT48JCvpj7q6dHdb7+tQ+GwzsRiMpLC/f16u7NT9zc26mRv73nnvvrqq3Xw4MFRXD2QGkQKSJJjx45p+/btes8Y9Q1xOaOftrQofJ4f7e09cUK1R4+ed+5p06bp+PHjjq0VsAWRApJk586dyszMVNOHHyri4Me/u1wu+Rz6QEXANrwnBSTJpZdeqtWrVys/N1cDb7whxWKOzGuMUX9/v+LxuCPzATbhlRSQJIsXL1YgENCkwkJd/YMffGF7VWmpMs5z5t/0nBzNzc8/79ydnZ3Ky8tzaqmANYgUkCQej0eTJ09WR0eHCqqqlP8nfzJoe2VJiR696ipN8HgSfzE9LpcKfD7972uuUfk5ESqqrk7896FDh3TZZZeN8h4AyceP+4Akcblcmjlzpvbt26deYzQpGFT43XcV6+5ObK8sKdG07Gxt+/hjnezt1fScHN02Y4YKznnPyTd1qiYFg3K5XOrr61NjY6MeffTRVOwWMKqIFJBEX/nKVxSNRvXhhx/qqquu0pSlS9Xxn/8p8z/vT7lcLs2eNEmzv+TDEb2TJqnkjjvk9fslSXv37lUgEND06dOTsQtAUvHjPiCJsrOz9b3vfU/PPfec+iUVff/7yv/Wt+TyDu/fi56cHE297TblXXutXB6PwuGwNm3apDvvvFNuPqUXYxB/qoEkcrlcuv766zVp0iRt2rRJ8YwMTfvhD1VUXa3MwsLzP87jUdb06Sq9+25NWbpU7sxM9ff366WXXtK0adM0e/bsJO4FkDz8uA9IMpfLpRUrVmjDhg265JJL9M1vflNTb71V/rlz9fs9e9TT0qJoKKR4NCpPTo4mTJumwIIFCixYoKyyssQn8e7Zs0eNjY368Y9/zO9JYcwiUkCSuVwulZWV6S/+4i/05JNPKh6P67rrrlPO7Nma+NWvKnbmjMzAgEw8LpfHI3dGhtzZ2XJ7vTLG6MyZM9q9e7f+5V/+RQ8//LC+wkfIYwxzmTS8vn8kElEgEFA4HJb/f948BtKNMUYffvihnnjiCU2fPl2VlZWaPXu2PB7PkOMHBgZ0+PBhvfLKK/rkk0+0atUqAoW0NdzncSIFpJAxRp2dndqxY4caGhqUm5urP/7jP1Z5ebmKi4tljNHx48f161//Wm+//bbOnDmjYDCob3/72yoqKiJQSFtECkgTxhjFYjF1d3dr//79+tWvfqXDhw/r+PHjcrlcKigo0Ne+9jVdd911mjt3rvLy8s77agtIF0QKAGCt4T6Pcwo6AMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArDWiSG3cuFFz586V3++X3+9XMBjUjh07Ett7e3tVU1OjgoIC5eTkqLq6Wh0dHYPmaGtrU1VVlbKzs1VYWKgHH3xQAwMDzuwNAGBMGVGkpk2bpvXr16upqUnvvvuuvvWtb+mmm25SS0uLJOmBBx7QK6+8oi1btqi+vl5Hjx7VLbfcknh8LBZTVVWV+vr6tGfPHr3wwgt6/vnntXbtWmf3CgAwNpiLNGnSJPOzn/3MdHV1mYyMDLNly5bEtkOHDhlJpqGhwRhjzPbt243b7TahUCgxZuPGjcbv95toNDrs7xkOh40kEw6HL3b5AIAUGO7z+AW/JxWLxbR582adPn1awWBQTU1N6u/vV0VFRWLMrFmzVFZWpoaGBklSQ0OD5syZo6KiosSYyspKRSKRxKuxoUSjUUUikUE3AMDYN+JINTc3KycnRz6fT/fcc4+2bt2q8vJyhUIhZWZmKi8vb9D4oqIihUIhSVIoFBoUqLPbz247n3Xr1ikQCCRupaWlI102ACANjThSl19+uQ4cOKDGxkbde++9Wr58uT744IPRWFvCmjVrFA6HE7f29vZR/X4AADt4R/qAzMxMXXbZZZKk+fPna9++fXryySd12223qa+vT11dXYNeTXV0dKi4uFiSVFxcrL179w6a7+zZf2fHDMXn88nn8410qQCANHfRvycVj8cVjUY1f/58ZWRkqK6uLrGttbVVbW1tCgaDkqRgMKjm5mZ1dnYmxtTW1srv96u8vPxilwIAGGNG9EpqzZo1Wrp0qcrKytTd3a1NmzZp9+7dev311xUIBLRixQqtXr1a+fn58vv9uu+++xQMBrVo0SJJ0pIlS1ReXq477rhDjz/+uEKhkB5++GHV1NTwSgkA8AUjilRnZ6fuvPNOHTt2TIFAQHPnztXrr7+ub3/725KkJ554Qm63W9XV1YpGo6qsrNQzzzyTeLzH49G2bdt07733KhgMauLEiVq+fLkee+wxZ/cKADAmuIwxJtWLGKlIJKJAIKBwOCy/35/q5QAARmi4z+Ncuw8AYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtS4qUuvXr5fL5dKqVasS9/X29qqmpkYFBQXKyclRdXW1Ojo6Bj2ura1NVVVVys7OVmFhoR588EENDAxczFIAAGPQBUdq3759+qd/+ifNnTt30P0PPPCAXnnlFW3ZskX19fU6evSobrnllsT2WCymqqoq9fX1ac+ePXrhhRf0/PPPa+3atRe+FwCAsclcgO7ubjNz5kxTW1trrr/+enP//fcbY4zp6uoyGRkZZsuWLYmxhw4dMpJMQ0ODMcaY7du3G7fbbUKhUGLMxo0bjd/vN9FodFjfPxwOG0kmHA5fyPIBACk23OfxC3olVVNTo6qqKlVUVAy6v6mpSf39/YPunzVrlsrKytTQ0CBJamho0Jw5c1RUVJQYU1lZqUgkopaWliG/XzQaVSQSGXQDAIx93pE+YPPmzXrvvfe0b9++L2wLhULKzMxUXl7eoPuLiooUCoUSYz4fqLPbz24byrp16/STn/xkpEsFAKS5Eb2Sam9v1/33369/+7d/04QJE0ZrTV+wZs0ahcPhxK29vT1p3xsAkDojilRTU5M6Ozt19dVXy+v1yuv1qr6+Xk899ZS8Xq+KiorU19enrq6uQY/r6OhQcXGxJKm4uPgLZ/ud/frsmHP5fD75/f5BNwDA2DeiSC1evFjNzc06cOBA4rZgwQItW7Ys8d8ZGRmqq6tLPKa1tVVtbW0KBoOSpGAwqObmZnV2dibG1NbWyu/3q7y83KHdAgCMBSN6Tyo3N1ezZ88edN/EiRNVUFCQuH/FihVavXq18vPz5ff7dd999ykYDGrRokWSpCVLlqi8vFx33HGHHn/8cYVCIT388MOqqamRz+dzaLcAAGPBiE+c+EOeeOIJud1uVVdXKxqNqrKyUs8880xiu8fj0bZt23TvvfcqGAxq4sSJWr58uR577DGnlwIASHMuY4xJ9SJGKhKJKBAIKBwO8/4UAKSh4T6Pc+0+AIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1vKlewIUwxkiSIpFIilcCALgQZ5+/zz6fn09aRurkyZOSpNLS0hSvBABwMbq7uxUIBM67PS0jlZ+fL0lqa2v70p0b7yKRiEpLS9Xe3i6/35/q5ViL4zQ8HKfh4TgNjzFG3d3dKikp+dJxaRkpt/uzt9ICgQB/CIbB7/dznIaB4zQ8HKfh4Tj9YcN5kcGJEwAAaxEpAIC10jJSPp9Pjz76qHw+X6qXYjWO0/BwnIaH4zQ8HCdnucwfOv8PAIAUSctXUgCA8YFIAQCsRaQAANYiUgAAa6VlpJ5++mlNnz5dEyZM0MKFC7V3795ULymp3nzzTX33u99VSUmJXC6XXn755UHbjTFau3atpk6dqqysLFVUVOg3v/nNoDGnTp3SsmXL5Pf7lZeXpxUrVqinpyeJezG61q1bp2uuuUa5ubkqLCzUzTffrNbW1kFjent7VVNTo4KCAuXk5Ki6ulodHR2DxrS1tamqqkrZ2dkqLCzUgw8+qIGBgWTuyqjauHGj5s6dm/jF02AwqB07diS2c4yGtn79erlcLq1atSpxH8dqlJg0s3nzZpOZmWn++Z//2bS0tJi7777b5OXlmY6OjlQvLWm2b99u/uZv/sa89NJLRpLZunXroO3r1683gUDAvPzyy+bXv/61+d73vmdmzJhhzpw5kxhzww03mHnz5pl33nnH/OpXvzKXXXaZuf3225O8J6OnsrLSPPfcc+bgwYPmwIED5jvf+Y4pKyszPT09iTH33HOPKS0tNXV1debdd981ixYtMn/0R3+U2D4wMGBmz55tKioqzP79+8327dvN5MmTzZo1a1KxS6Piv/7rv8yrr75q/vu//9u0traav/7rvzYZGRnm4MGDxhiO0VD27t1rpk+fbubOnWvuv//+xP0cq9GRdpG69tprTU1NTeLrWCxmSkpKzLp161K4qtQ5N1LxeNwUFxebDRs2JO7r6uoyPp/PvPjii8YYYz744AMjyezbty8xZseOHcblcplPPvkkaWtPps7OTiPJ1NfXG2M+OyYZGRlmy5YtiTGHDh0ykkxDQ4Mx5rN/DLjdbhMKhRJjNm7caPx+v4lGo8ndgSSaNGmS+dnPfsYxGkJ3d7eZOXOmqa2tNddff30iUhyr0ZNWP+7r6+tTU1OTKioqEve53W5VVFSooaEhhSuzx5EjRxQKhQYdo0AgoIULFyaOUUNDg/Ly8rRgwYLEmIqKCrndbjU2NiZ9zckQDocl/f+LEzc1Nam/v3/QcZo1a5bKysoGHac5c+aoqKgoMaayslKRSEQtLS1JXH1yxGIxbd68WadPn1YwGOQYDaGmpkZVVVWDjonEn6fRlFYXmD1x4oRisdig/8mSVFRUpMOHD6doVXYJhUKSNOQxOrstFAqpsLBw0Hav16v8/PzEmLEkHo9r1apV+vrXv67Zs2dL+uwYZGZmKi8vb9DYc4/TUMfx7Laxorm5WcFgUL29vcrJydHWrVtVXl6uAwcOcIw+Z/PmzXrvvfe0b9++L2zjz9PoSatIAReipqZGBw8e1FtvvZXqpVjp8ssv14EDBxQOh/Uf//EfWr58uerr61O9LKu0t7fr/vvvV21trSZMmJDq5YwrafXjvsmTJ8vj8XzhjJmOjg4VFxenaFV2OXscvuwYFRcXq7Ozc9D2gYEBnTp1aswdx5UrV2rbtm3atWuXpk2blri/uLhYfX196urqGjT+3OM01HE8u22syMzM1GWXXab58+dr3bp1mjdvnp588kmO0ec0NTWps7NTV199tbxer7xer+rr6/XUU0/J6/WqqKiIYzVK0ipSmZmZmj9/vurq6hL3xeNx1dXVKRgMpnBl9pgxY4aKi4sHHaNIJKLGxsbEMQoGg+rq6lJTU1NizM6dOxWPx7Vw4cKkr3k0GGO0cuVKbd26VTt37tSMGTMGbZ8/f74yMjIGHafW1la1tbUNOk7Nzc2Dgl5bWyu/36/y8vLk7EgKxONxRaNRjtHnLF68WM3NzTpw4EDitmDBAi1btizx3xyrUZLqMzdGavPmzcbn85nnn3/efPDBB+ZHP/qRycvLG3TGzFjX3d1t9u/fb/bv328kmb//+783+/fvN7/73e+MMZ+dgp6Xl2d++ctfmvfff9/cdNNNQ56CftVVV5nGxkbz1ltvmZkzZ46pU9DvvfdeEwgEzO7du82xY8cSt08//TQx5p577jFlZWVm586d5t133zXBYNAEg8HE9rOnDC9ZssQcOHDAvPbaa2bKlClj6pThhx56yNTX15sjR46Y999/3zz00EPG5XKZN954wxjDMfoynz+7zxiO1WhJu0gZY8w//MM/mLKyMpOZmWmuvfZa884776R6SUm1a9cuI+kLt+XLlxtjPjsN/ZFHHjFFRUXG5/OZxYsXm9bW1kFznDx50tx+++0mJyfH+P1+c9ddd5nu7u4U7M3oGOr4SDLPPfdcYsyZM2fMX/7lX5pJkyaZ7Oxs8/3vf98cO3Zs0Dy//e1vzdKlS01WVpaZPHmy+fGPf2z6+/uTvDej54c//KG55JJLTGZmppkyZYpZvHhxIlDGcIy+zLmR4liNDj6qAwBgrbR6TwoAML4QKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYK3/B0l4qrgpgcvoAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "-116.62361788537898"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test(play=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Gym",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
